﻿
#include <boost/program_options.hpp>
#include <boost/scoped_ptr.hpp>
#include <boost/filesystem.hpp>
#include <iostream>
#include <iomanip>
#include <iterator>
#include <fstream>
#include <sstream>

#include <kodes.h>

using namespace std;


//#define READ_TATOEBA
//#define MAKE_UNA_OFTECO
//#define MAKE_DUA_OFTECO
#define MAKE_U_UX_TABLE
#define CLEAR_U_UX_TABLE

#define RESULT_FILE_NAME "res.txt"
#define PATH_TO_SRC_TEXT_FILES "E:\\sendoj\\eogram\\tte\\"

#ifdef MAKE_UNA_OFTECO
std::map<wchar_t, unsigned long> ofteco;
#endif

#ifdef MAKE_DUA_OFTECO
std::map<pair<wchar_t,wchar_t>, unsigned long> duofteco;
#endif

#ifdef MAKE_U_UX_TABLE
std::map<pair<wchar_t,pair<wchar_t,wchar_t> >, unsigned long> triofteco;
#endif


#ifdef READ_TATOEBA
void convertTatoebaSentences()
{
  ifstream ifs(PATH_TO_SRC_TEXT_FILES "sentences.csv", ios_base::in | ios_base::binary);
  ofstream ofs(PATH_TO_SRC_TEXT_FILES "epo_sentences.txt", ios_base::out | ios_base::binary);
  ofstream ofs_err(PATH_TO_SRC_TEXT_FILES "eepo_sentences.txt", ios_base::out | ios_base::binary);

  LegoKonverto *lk = LegoKonverto::getBySignaro(S_UTF8);
  lk->setStream(&ifs);

  ScriboKonverto *sk = ScriboKonverto::getBySignaro(S_UTF8);
  sk->setStream(&ofs);

  ScriboKonverto *sk_err = ScriboKonverto::getBySignaro(S_UTF8);
  sk_err->setStream(&ofs_err);

  int counter = 0;
  bool haveErr = false;
  wstring line;
  for(;;)
  {
    wchar_t c = lk->doni();
    if (c == 0)
    {
      delete lk;
      delete sk;
      delete sk_err;
      ifs.close();
      ofs.close();
      ofs_err.close();
      return;
    }
    else if (c == '\n')
    {
      size_t pos = line.find(L"\tepo\t");
      if (pos != string::npos)
      {
        if (haveErr)
        {
          for (auto it = line.begin(); it!=line.end(); ++it)
            sk_err->meti(*it);
          sk_err->meti('\n');
        }
        else
        {
          for (auto it = line.begin()+(pos+5); it!=line.end(); ++it)
            sk->meti(*it);
          sk->meti('\n');
        }
      }
      line = wstring();
      haveErr = false;

      if (++counter >= 20510)
      {
        counter = 0;
        cout << '.';
      }
    }
    else
    {
      line += c;
      const static wstring tmp(L"\t AaBbCcĈĉDdEeFfGgĜĝHhĤĥIiJjĴĵKkLlMmNnOoPpRrSsŜŝTtUuŬŭVvZz.,:;!?0123456789'\"-—+*/\\()«»„“");
      if (!haveErr && wstring::npos == tmp.find_first_of(c))
        haveErr = true;
    }
  }
}
#endif

//#ifdef MAKE_U_UX_TABLE
//void createUTable(std::map<pair<wchar_t,pair<wchar_t,wchar_t> >, unsigned long>& m)
//{
//  const static wstring abc (L"AaBbCcĈĉDdEeFfGgĜĝHhĤĥIiJjĴĵKkLlMmNnOoPpRrSsŜŝTtUuŬŭVvZz");
//
//  for (auto it=m.begin(); it != m.end(); ++it)
//  {
//  }
//
//}
//#endif // MAKE_U_UX_TABLE


void filterChar(wchar_t &c)
{
#ifdef MAKE_U_UX_TABLE
  const static wstring abc (L"AaBbCcĈĉDdEeFfGgĜĝHhĤĥIiJjĴĵKkLlMmNnOoPpRrSsŜŝTtUuŬŭVvZz");
  if (wstring::npos == abc.find_first_of(c))
    c = L' ';
#else
  const static wstring wrongs(L".,:;!?0123456789");
  if (unsigned short(c) < 0x20 ||  wstring::npos != wrongs.find_first_of(c))
    c = L' ';
#endif
}



#ifdef MAKE_U_UX_TABLE
void putTableLine(wostringstream &oss, wchar_t c1, wchar_t c2, wchar_t c3, unsigned long oft_u, unsigned long oft_ux)
{
  if (!oft_ux || oft_ux < oft_u)
  {
#ifdef CLEAR_U_UX_TABLE
    return;
#endif
    oss << L"//  ";
  }
  else
    oss << L"    ";

  oss << L"{" << setw(4) << (int)c1 << "," << setw(4) << (int)c3 << " }, // ",
  oss << c1 << L"_" << c3 << L"  (u)=" << oft_u << L", (ux)=" << oft_ux << endl;
}
#endif // MAKE_U_UX_TABLE

int main(int ac, char* av[])
{

#ifdef READ_TATOEBA
  convertTatoebaSentences();
  return 0;
#endif

  cout << "Kolektas la statistikon." << endl;

  try
  {

    namespace po = boost::program_options;
    po::options_description desc("Allowed options");
    desc.add_options()
      ("help", "produce help message")
      ("dir", po::value<string>(), "directory with source files");


    po::positional_options_description p;
    p.add("dir", -1);

    po::variables_map vm;
    po::store(po::command_line_parser(ac, av).options(desc).positional(p).run(), vm);
    po::notify(vm);





    LegoKonverto *legilo = LegoKonverto::getBySignaro(S_UTF8);
    boost::scoped_ptr<LegoKonverto> memforigi(legilo);


    using namespace boost::filesystem;

//    directory_iterator dir_iter(vm["dir"].as<string>());
    directory_iterator dir_iter(PATH_TO_SRC_TEXT_FILES);
    directory_iterator iter_end;

    for (; dir_iter != iter_end; ++dir_iter)
    {
      if (is_regular_file(dir_iter->status()))
      {
        string path = dir_iter->path().string(); //.filename();


        ifstream ifs(path);
        if (!ifs.is_open())
        {
          cerr << "error opening " << path << endl;
          continue;
        }

        legilo->setStream(&ifs);

        wstring src;
        wcout << L'.';
        legilo->doni(src);
        wcout << L'.';

        wchar_t prv = 0;
        wchar_t prvprv = 0;
        for (auto it=src.begin(); it!=src.end(); ++it)
        {
          wchar_t c = *it;
          filterChar(c);

#ifdef MAKE_UNA_OFTECO
          ofteco[c]++;
#endif
#ifdef MAKE_DUA_OFTECO
          if (prv)
            duofteco[make_pair(prv,c)]++;
#endif
#ifdef MAKE_U_UX_TABLE
          if (prv && prvprv && (prv == L'U' || prv == L'u' || prv == L'Ŭ' || prv == L'ŭ'))
            triofteco[make_pair(prvprv,make_pair(c,prv))]++;
#endif
          prvprv = prv;
          prv = c;
        }

      }
    }

    wcout << L'\n';

    wostringstream oss;

#ifdef MAKE_UNA_OFTECO
    float totalChars = 0.0f;
    for (auto it = ofteco.begin(); it != ofteco.end(); ++it)
    {
      if (it->first == ' ')
        continue;
      totalChars += it->second;
    }

    oss << L"\n\nchar\tcount: \t%\n";
    oss << fixed << setprecision(3);
    for (auto it = ofteco.begin(); it != ofteco.end(); ++it)
    {
      wchar_t c = it->first;
      if (c == ' ')
        continue;
      unsigned long count = it->second;
      float percent = count/totalChars;
      oss << c << L"\t" << count << L"\t" << percent << L"\n";
    }
#endif // MAKE_UNA_OFTECO





#ifdef MAKE_DUA_OFTECO
    float total = 0.0f;
    for (auto it = duofteco.begin(); it != duofteco.end(); ++it)
    {
      if (it->first.first == ' ' && it->first.second == ' ')
        continue;
      total += it->second;
    }

    oss << L"chars\tcount: \t%\n";
    oss << fixed << setprecision(3);
    for (auto it = duofteco.begin(); it != duofteco.end(); ++it)
    {
      wchar_t c1 = it->first.first;
      wchar_t c2 = it->first.second;
      if (c1 == ' ' && c2 == ' ')
        continue;
      unsigned long count = it->second;
      float percent = count/total;
      oss << c1 << c2 << L"\t" << count << L"\t" << percent << L"\n";
    }
#endif // MAKE_DUA_OFTECO




#ifdef MAKE_U_UX_TABLE
/*
    oss << L"chars\tcount\n";
    while (!triofteco.empty())
    {
      auto mx = triofteco.begin();
      for (auto it = triofteco.begin(); it != triofteco.end(); ++it)
      {
        if (it->second > mx->second)
          mx = it;
      }

      unsigned long count = mx->second;
      oss << (wchar_t)mx->first.first << (wchar_t)mx->first.second.first << (wchar_t)mx->first.second.second;
      oss << L"\t" << count << L"\t\n";

      triofteco.erase(mx);
    }
*/
/*
    oss << L"\n\nchars\tcount\n";
    for (auto it = triofteco.begin(); it != triofteco.end(); ++it)
    {
      unsigned long count = it->second;
      oss << (wchar_t)it->first.first << (wchar_t)it->first.second.first << (wchar_t)it->first.second.second;
      oss << L"\t" << count << L"\t\n";
    }
*/


  const static wstring abc (L" AaBbCcĈĉDdEeFfGgĜĝHhĤĥIiJjĴĵKkLlMmNnOoPpRrSsŜŝTtUuŬŭVvZz");


  oss << L"// auto generated file, do nod edit manualy\n\n"
    L"struct {\n"
    L"  wchar_t c1;\n"
    L"  wchar_t c3;\n"
    L"} kie_ux[] = {\n";

  for (auto it1 = abc.begin(); it1 != abc.end(); ++it1)
    for (auto it2 = abc.begin(); it2 != abc.end(); ++it2)
    {
      wchar_t c1 = *it1;
      wchar_t c3 = *it2;

      auto res_u  = triofteco.find(make_pair(c1,make_pair(c3,L'u')));
      auto res_ux = triofteco.find(make_pair(c1,make_pair(c3,L'ŭ')));

      unsigned long oft_u = res_u == triofteco.end() ? 0 : res_u->second;
      unsigned long oft_ux = res_ux == triofteco.end() ? 0 : res_ux->second;

      putTableLine(oss, c1, L'u', c3, oft_u, oft_ux);
    }

  oss << L"};\n\n"
    L"struct {\n"
    L"  wchar_t c1;\n"
    L"  wchar_t c3;\n"
    L"} kie_UX[] = {\n";

  for (auto it1 = abc.begin(); it1 != abc.end(); ++it1)
    for (auto it2 = abc.begin(); it2 != abc.end(); ++it2)
    {
      wchar_t c1 = *it1;
      wchar_t c3 = *it2;

      auto res_u  = triofteco.find(make_pair(c1,make_pair(c3,L'U')));
      auto res_ux = triofteco.find(make_pair(c1,make_pair(c3,L'Ŭ')));

      unsigned long oft_u = res_u == triofteco.end() ? 0 : res_u->second;
      unsigned long oft_ux = res_ux == triofteco.end() ? 0 : res_ux->second;

      putTableLine(oss, c1, L'u', c3, oft_u, oft_ux);
    }

  oss << L"};\n";


#endif // MAKE_U_UX_TABLE



    cout << "\n results saved in ";
    ofstream wrFile(RESULT_FILE_NAME, ios_base::out | ios_base::binary);
    if (!wrFile.is_open())
      throw runtime_error("Can not open output file");

    ScriboKonverto *ScrKonv = ScriboKonverto::getBySignaro(S_UTF8);
    ScrKonv->setStream(&wrFile);
    ScrKonv->meti(oss.str());
    wrFile.close();
    delete ScrKonv;
    cout << RESULT_FILE_NAME << "\n";
  }
  catch(exception& e)
  {
    cerr << "error: " << e.what() << "\n";
    return 1;
  }
  return 0;
}

